{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "12f4dced",
   "metadata": {},
   "source": [
    "双模型使用两个不同的模型计算value和target,缓解了自举造成的过高估计."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAT0ElEQVR4nO3db0xb570H8K8N2Al/jj1g2EVgJdJ6l3DzpxtJ4KQvNjVeWIeiZeVFN0Udi6JUzUzUlCrakNr033qpshfduqXkzZb0xbLsMo1NRWk7RlpypzghoeOKQMM6KRUsxXZTLsdAgm3s331RcVYnJAMCfnD8/UhH4jzPY/t3jvHX54+PbRERARGRAlbVBRBR5mIAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMsoC6MiRI1i1ahVWrFiBqqoqdHd3qyqFiBRREkC/+93v0NjYiOeeew7vv/8+Nm7ciJqaGoRCIRXlEJEiFhUXo1ZVVWHz5s345S9/CQBIJBIoLy/H/v378eMf/zjV5RCRItmpfsBoNIqenh40NTWZbVarFV6vF36/f9bbRCIRRCIRcz6RSGB0dBRFRUWwWCxLXjMRzY+IYHx8HKWlpbBab7+jlfIAunbtGuLxOFwuV1K7y+XC5cuXZ71Nc3MzXnjhhVSUR0SLaHh4GGVlZbftT3kALURTUxMaGxvNecMw4PF4MDw8DE3TFFZGRLMJh8MoLy9HQUHBHcelPICKi4uRlZWFYDCY1B4MBuF2u2e9jd1uh91uv6Vd0zQGENEy9u8OkaT8LJjNZkNlZSU6OzvNtkQigc7OTui6nupyiEghJbtgjY2NqK+vx6ZNm7Blyxb87Gc/w+TkJHbv3q2iHCJSREkAPfroo/jkk09w6NAhBAIBPPDAA3j77bdvOTBNRPc2JZ8DulvhcBgOhwOGYfAYENEyNNfXKK8FIyJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTLzDqAzZ85gx44dKC0thcViwR//+MekfhHBoUOHcN9992HlypXwer348MMPk8aMjo5i165d0DQNTqcTe/bswcTExF0tCBGln3kH0OTkJDZu3IgjR47M2n/48GG89tprOHr0KM6fP4+8vDzU1NRgamrKHLNr1y709/ejo6MD7e3tOHPmDB5//PGFLwURpSe5CwCkra3NnE8kEuJ2u+WnP/2p2TY2NiZ2u11++9vfiojIwMCAAJALFy6YY9566y2xWCxy9erVOT2uYRgCQAzDuJvyiWiJzPU1uqjHgK5cuYJAIACv12u2ORwOVFVVwe/3AwD8fj+cTic2bdpkjvF6vbBarTh//vys9xuJRBAOh5MmIkp/ixpAgUAAAOByuZLaXS6X2RcIBFBSUpLUn52djcLCQnPMzZqbm+FwOMypvLx8McsmIkXS4ixYU1MTDMMwp+HhYdUlEdEiWNQAcrvdAIBgMJjUHgwGzT63241QKJTUPz09jdHRUXPMzex2OzRNS5qIKP0tagCtXr0abrcbnZ2dZls4HMb58+eh6zoAQNd1jI2Noaenxxxz+vRpJBIJVFVVLWY5RLTMZc/3BhMTE/jHP/5hzl+5cgW9vb0oLCyEx+PBgQMH8JOf/AT3338/Vq9ejWeffRalpaXYuXMnAGDt2rX45je/ib179+Lo0aOIxWJoaGjAd7/7XZSWli7aghFRGpjv6bV3331XANwy1dfXi8hnp+KfffZZcblcYrfbZdu2bTI4OJh0H59++ql873vfk/z8fNE0TXbv3i3j4+OLfoqPiNSY62vUIiKiMP8WJBwOw+FwwDAMHg8iWobm+hpNi7NgRHRvYgARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREysz7Z3mIFouIYGosgOvXhsy2LHseHOX/CYvForAyShUGECk19tH/4p/dfzDnc4s9cJStBSxZCquiVOEuGCmVSEyrLoEUYgCRUpKIqy6BFGIAkVIS5xZQJmMAkVLcAspsDCBSSCA8BpTRGECkVCLOLaBMNq8Aam5uxubNm1FQUICSkhLs3LkTg4ODSWOmpqbg8/lQVFSE/Px81NXVIRgMJo0ZGhpCbW0tcnNzUVJSgoMHD2J6mu+EGUeAePR6UpM126aoGFJhXgHU1dUFn8+Hc+fOoaOjA7FYDNu3b8fk5KQ55qmnnsKbb76J1tZWdHV14eOPP8Yjjzxi9sfjcdTW1iIajeLs2bN44403cPz4cRw6dGjxlorSgkgCU2PJb04rnPcB/BBi5pC7EAqFBIB0dXWJiMjY2Jjk5ORIa2urOeaDDz4QAOL3+0VE5NSpU2K1WiUQCJhjWlpaRNM0iUQic3pcwzAEgBiGcTflk2Lx6Zj0/ffz0n10rzl99D+/kUQirro0uktzfY3e1TEgwzAAAIWFhQCAnp4exGIxeL1ec8yaNWvg8Xjg9/sBAH6/H+vXr4fL5TLH1NTUIBwOo7+/f9bHiUQiCIfDSRPdmyzWbADcAsoUCw6gRCKBAwcO4MEHH8S6desAAIFAADabDU6nM2msy+VCIBAwx3w+fGb6Z/pm09zcDIfDYU7l5eULLZuWOYuVl2BkkgUHkM/nw6VLl3Dy5MnFrGdWTU1NMAzDnIaHh5f8MUkNSxYvT8wkC3q2Gxoa0N7ejjNnzqCsrMxsd7vdiEajGBsbS9oKCgaDcLvd5pju7u6k+5s5SzYz5mZ2ux12u30hpVKasXILKKPMawtIRNDQ0IC2tjacPn0aq1evTuqvrKxETk4OOjs7zbbBwUEMDQ1B13UAgK7r6OvrQygUMsd0dHRA0zRUVFTczbLQPYBbQJllXs+2z+fDiRMn8Kc//QkFBQXmMRuHw4GVK1fC4XBgz549aGxsRGFhITRNw/79+6HrOqqrqwEA27dvR0VFBR577DEcPnwYgUAAzzzzDHw+H7dyiMeAMsy8AqilpQUA8PWvfz2p/dixY/jBD34AAHj11VdhtVpRV1eHSCSCmpoavP766+bYrKwstLe3Y9++fdB1HXl5eaivr8eLL754d0tCaUhuabFY+OH8TGIRkVv/C5a5cDgMh8MBwzCgaZrqcmiB4rEIBv7wX5gaGzHbPFsfhWv9NoVV0WKY62uUbzekjiQ+myhjMYBIGZEE0nADnBYRA4iUkUQCwi2gjMYAImWEu2AZjwFE6iS4C5bpGECkjCTiwM1fycqv4sgoDCBSJnbDQGxqwpy3ZGVjhcN1h1vQvYYBRMpI4uZjQBZ+I2KGYQDRsmGxWHgpRoZhANGywgDKLAwgWka4BZRpGEC0fFi4BZRpGEC0jHALKNMwgGjZ4EHozMMAInVm+RS0xcp/yUzCZ5uUSfB34TMeA4iUEf4ufMZjAJEywi2gjMcAImUSN1+IShmHAUTKSJxbQJmOAUTKcBeMGECkDA9CEwOIlJkKh5Lmc3IdsGbx6zgyCQOIlIlHrifNZ9lz+dPMGYbPNi2ZaDSK69ev37H/8+JxgWEYt/1SspUrV/Lnu+8xDCBaMm1tbXj66adv2+/bsR7bNpab8+cvXMDLjS2IxWf/pYyXXnoJu3fvXvQ6SR0GEC2ZyclJXL169bb9E5P/gSs3NuCTmAdFOVcxMXkK/7x6FdO3CaCJiYlZ2yl9zesYUEtLCzZs2ABN06BpGnRdx1tvvWX2T01NwefzoaioCPn5+airq0MwGEy6j6GhIdTW1iI3NxclJSU4ePAgpqd5OjYTfXRjHf5+fQtGY6X48Hol/j65nj/Tk2HmFUBlZWV45ZVX0NPTg4sXL+Khhx7Ct7/9bfT39wMAnnrqKbz55ptobW1FV1cXPv74YzzyyCPm7ePxOGpraxGNRnH27Fm88cYbOH78OA4dOrS4S0VpYTLugJj/glYYUcdsF8jTPWxeu2A7duxImn/55ZfR0tKCc+fOoaysDL/61a9w4sQJPPTQQwCAY8eOYe3atTh37hyqq6vx5z//GQMDA/jLX/4Cl8uFBx54AC+99BJ+9KMf4fnnn4fNxlOwmcRtu4Icyw3EZAVyLBGUZH+IBBMooyz4GFA8HkdraysmJyeh6zp6enoQi8Xg9XrNMWvWrIHH44Hf70d1dTX8fj/Wr18Pl+tfv/1UU1ODffv2ob+/H1/5ylfmVcPly5eRn5+/0EWgJTYyMnLH/r5LZxC4FsT/xdxw5oQwdm3wjuMDgQAGBgYWs0RaInM9XjfvAOrr64Ou65iamkJ+fj7a2tpQUVGB3t5e2Gw2OJ3OpPEulwuBQADAZ/9Anw+fmf6ZvtuJRCKIRCLmfDgcBgAYhsHjR8vYnU7BA8DZ/mGgf3jO9zc1NYWxsbG7rIpSYXJyck7j5h1AX/7yl9Hb2wvDMPD73/8e9fX16OrqmneB89Hc3IwXXnjhlvaqqipomrakj00Ld/ny5UW9v1WrVmHr1q2Lep+0NGY2Ev6deX8S2maz4Utf+hIqKyvR3NyMjRs34uc//zncbjei0egt71DBYBButxsA4Ha7bzkrNjM/M2Y2TU1NMAzDnIaH5/6uSUTL111fipFIJBCJRFBZWYmcnBx0dnaafYODgxgaGoKu6wAAXdfR19eHUOhf1wB1dHRA0zRUVFTc9jHsdrt56n9mIqL0N69dsKamJjz88MPweDwYHx/HiRMn8N577+Gdd96Bw+HAnj170NjYiMLCQmiahv3790PXdVRXVwMAtm/fjoqKCjz22GM4fPgwAoEAnnnmGfh8Pn7EnigDzSuAQqEQvv/972NkZAQOhwMbNmzAO++8g2984xsAgFdffRVWqxV1dXWIRCKoqanB66+/bt4+KysL7e3t2LdvH3RdR15eHurr6/Hiiy8u7lLRsmCz2RZ1a5VvUvcei6ThR0/D4TAcDgcMw+Du2DI2OTmJ0dHRRbs/p9OJgoKCRbs/WjpzfY3yWjBaMnl5ecjLy1NdBi1j/D4gIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpEy26gIWQkQAAOFwWHElRDSbmdfmzGv1dtIygD799FMAQHl5ueJKiOhOxsfH4XA4btuflgFUWFgIABgaGrrjwlGycDiM8vJyDA8PQ9M01eWkBa6zhRERjI+Po7S09I7j0jKArNbPDl05HA7+UyyApmlcb/PEdTZ/c9k44EFoIlKGAUREyqRlANntdjz33HOw2+2qS0krXG/zx3W2tCzy786TEREtkbTcAiKiewMDiIiUYQARkTIMICJSJi0D6MiRI1i1ahVWrFiBqqoqdHd3qy5JmebmZmzevBkFBQUoKSnBzp07MTg4mDRmamoKPp8PRUVFyM/PR11dHYLBYNKYoaEh1NbWIjc3FyUlJTh48CCmp6dTuSjKvPLKK7BYLDhw4IDZxnWWIpJmTp48KTabTX79619Lf3+/7N27V5xOpwSDQdWlKVFTUyPHjh2TS5cuSW9vr3zrW98Sj8cjExMT5pgnnnhCysvLpbOzUy5evCjV1dWydetWs396elrWrVsnXq9X/va3v8mpU6ekuLhYmpqaVCxSSnV3d8uqVatkw4YN8uSTT5rtXGepkXYBtGXLFvH5fOZ8PB6X0tJSaW5uVljV8hEKhQSAdHV1iYjI2NiY5OTkSGtrqznmgw8+EADi9/tFROTUqVNitVolEAiYY1paWkTTNIlEIqldgBQaHx+X+++/Xzo6OuRrX/uaGUBcZ6mTVrtg0WgUPT098Hq9ZpvVaoXX64Xf71dY2fJhGAaAf12w29PTg1gslrTO1qxZA4/HY64zv9+P9evXw+VymWNqamoQDofR39+fwupTy+fzoba2NmndAFxnqZRWF6Neu3YN8Xg86UkHAJfLhcuXLyuqavlIJBI4cOAAHnzwQaxbtw4AEAgEYLPZ4HQ6k8a6XC4EAgFzzGzrdKbvXnTy5Em8//77uHDhwi19XGepk1YBRHfm8/lw6dIl/PWvf1VdyrI2PDyMJ598Eh0dHVixYoXqcjJaWu2CFRcXIysr65azEcFgEG63W1FVy0NDQwPa29vx7rvvoqyszGx3u92IRqMYGxtLGv/5deZ2u2ddpzN995qenh6EQiF89atfRXZ2NrKzs9HV1YXXXnsN2dnZcLlcXGcpklYBZLPZUFlZic7OTrMtkUigs7MTuq4rrEwdEUFDQwPa2tpw+vRprF69Oqm/srISOTk5SetscHAQQ0ND5jrTdR19fX0IhULmmI6ODmiahoqKitQsSApt27YNfX196O3tNadNmzZh165d5t9cZymi+ij4fJ08eVLsdrscP35cBgYG5PHHHxen05l0NiKT7Nu3TxwOh7z33nsyMjJiTtevXzfHPPHEE+LxeOT06dNy8eJF0XVddF03+2dOKW/fvl16e3vl7bffli9+8YsZdUr582fBRLjOUiXtAkhE5Be/+IV4PB6x2WyyZcsWOXfunOqSlAEw63Ts2DFzzI0bN+SHP/yhfOELX5Dc3Fz5zne+IyMjI0n389FHH8nDDz8sK1eulOLiYnn66aclFouleGnUuTmAuM5Sg1/HQUTKpNUxICK6tzCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZf4fEiWli5rSG7gAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ecfbe912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       " ),\n",
       " Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型,评估状态下每个动作的价值\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "#延迟更新的模型,用于计算target\n",
    "model_delay = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "#复制参数\n",
    "model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "model, model_delay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\appDir\\python3.10\\lib\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  if not isinstance(terminated, (bool, np.bool8)):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-987.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "        if random.random() < 0.1:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bfcfd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_17240\\2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(200,\n",
       " (array([-0.03957091,  0.03456768,  0.03941284,  0.02746275], dtype=float32),\n",
       "  0,\n",
       "  1.0,\n",
       "  array([-0.03887955, -0.16109666,  0.0399621 ,  0.3323157 ], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 405 -990.45\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(100):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                target = model_delay(next_state)\n",
    "            target = target.max(dim=1)[0].reshape(-1, 1)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        #复制参数\n",
    "        if (epoch + 1) % 5 == 0:\n",
    "            model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVo0lEQVR4nO3dfWxb5b0H8K8dv+T1OE3S2A2J11yBVkJfGGmbekx308gaWDStI7raUMcyVBXRORUlU+9dJCiCi5Sq/MFgK0G601r+YZ0y0SFCAUUppNrFtDQQ3TSk4X0JbW3TlBznpbEd+3f/6HKKm6SNHTdP034/0pE4z/PLOc95ir+yz4ttEhEBEZECZtUDIKIbFwOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUURZAe/fuxfLly5GZmYmqqiocO3ZM1VCISBElAfTXv/4VjY2NePzxx/H+++9jzZo1qKmpQTAYVDEcIlLEpOJh1KqqKqxbtw5//OMfAQDxeBxlZWXYvn07fve73y30cIhIEctC7zASiaCrqwtNTU1Gm9lsRnV1NXw+34x/Ew6HEQ6HjfV4PI5z586hsLAQJpPpqo+ZiJIjIhgZGUFJSQnM5tk/aC14AJ09exaxWAxOpzOh3el04uTJkzP+TXNzM5544omFGB4RpdHg4CBKS0tn7V/wAEpFU1MTGhsbjXVd1+F2uzE4OAhN0xSOjIhmEgqFUFZWhry8vMvWLXgAFRUVISMjA4FAIKE9EAjA5XLN+Dd2ux12u31au6ZpDCCia9iVTpEs+FUwm82GyspKdHR0GG3xeBwdHR3weDwLPRwiUkjJR7DGxkbU19dj7dq1WL9+PX7/+99jbGwMDzzwgIrhEJEiSgLo5z//Ob766ivs2rULfr8ft99+O954441pJ6aJ6Pqm5D6g+QqFQnA4HNB1neeAiK5Bc32N8lkwIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKZN0AB05cgQ/+clPUFJSApPJhL///e8J/SKCXbt2YdmyZcjKykJ1dTU+/vjjhJpz585h8+bN0DQN+fn52LJlC0ZHR+d1IES0+CQdQGNjY1izZg327t07Y/+ePXvw3HPP4YUXXsDRo0eRk5ODmpoaTExMGDWbN29Gb28v2tvb0dbWhiNHjuDBBx9M/SiIaHGSeQAgBw8eNNbj8bi4XC55+umnjbbh4WGx2+3yl7/8RUREPvzwQwEg7733nlHz+uuvi8lkklOnTs1pv7quCwDRdX0+wyeiq2Sur9G0ngP6/PPP4ff7UV1dbbQ5HA5UVVXB5/MBAHw+H/Lz87F27Vqjprq6GmazGUePHp1xu+FwGKFQKGEhosUvrQHk9/sBAE6nM6Hd6XQafX6/H8XFxQn9FosFBQUFRs2lmpub4XA4jKWsrCydwyYiRRbFVbCmpiboum4sg4ODqodERGmQ1gByuVwAgEAgkNAeCASMPpfLhWAwmNA/OTmJc+fOGTWXstvt0DQtYSGixS+tAVReXg6Xy4WOjg6jLRQK4ejRo/B4PAAAj8eD4eFhdHV1GTWHDx9GPB5HVVVVOodDRNc4S7J/MDo6ik8++cRY//zzz9Hd3Y2CggK43W7s2LEDTz31FG655RaUl5fjscceQ0lJCTZt2gQAuPXWW3H33Xdj69ateOGFFxCNRtHQ0IBf/OIXKCkpSduBEdEikOzltbfeeksATFvq6+tF5MKl+Mcee0ycTqfY7Xa56667pL+/P2EbQ0NDct9990lubq5omiYPPPCAjIyMpP0SHxGpMdfXqElERGH+pSQUCsHhcEDXdZ4PIroGzfU1uiiughHR9YkBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZZL+XTAiIonHEezrhMWeg6yCm2DLWYIMWxZMJlNS22EAEVHSImNf48z7hxAdD8FstSNTW4qb7/bCnluQ1Hb4EYyIkhYZPYfo+REAgnh0AvFYFBkWW9LbYQARUVJEBPqXHwISN9oyHU5k2HOS3hYDiIiSI3FERoYSmrTSW1PaFAOIiJISm4xg1P/JxQaTGfa8oqRPQAMMICJKUlgPIjoxaqxbM3ORvfRbKW0rqQBqbm7GunXrkJeXh+LiYmzatAn9/f0JNRMTE/B6vSgsLERubi7q6uoQCAQSagYGBlBbW4vs7GwUFxdj586dmJycTOkAiGjhiAgm9ADi0Qmjze4ohsWWldL2kgqgzs5OeL1evPvuu2hvb0c0GsXGjRsxNjZm1DzyyCN49dVX0drais7OTpw+fRr33nuv0R+LxVBbW4tIJIJ33nkHL774Ivbv349du3aldABEtLBG/Z8mrGcXlcGUYU1tYzIPwWBQAEhnZ6eIiAwPD4vVapXW1lajpq+vTwCIz+cTEZFDhw6J2WwWv99v1LS0tIimaRIOh+e0X13XBYDouj6f4RNRkmKTUel75Wk59sLWfy0Pylcn/3da3Vxfo/M6B6TrOgCgoODCzUddXV2IRqOorq42alasWAG32w2fzwcA8Pl8WLVqFZxOp1FTU1ODUCiE3t7eGfcTDocRCoUSFiJaeNGxYYwPDRrrpgwLsovcKW8v5QCKx+PYsWMH7rzzTqxcuRIA4Pf7YbPZkJ+fn1DrdDrh9/uNmm+Gz1T/VN9Mmpub4XA4jKWsrCzVYRPRPETGhxGfjBrrmY5i2POKUt5eygHk9Xpx4sQJHDhwIOWdz1VTUxN0XTeWwcHBK/8REaWViGDk9EeQ+MULRtZsB8zW5O+AnpLSs2ANDQ1oa2vDkSNHUFpaarS7XC5EIhEMDw8nvAsKBAJwuVxGzbFjxxK2N3WVbKrmUna7HXa7PZWhElG6SBxjX32R0OQouw1A8vf/TEnqHZCIoKGhAQcPHsThw4dRXl6e0F9ZWQmr1YqOjg6jrb+/HwMDA/B4PAAAj8eDnp4eBINBo6a9vR2apqGioiLlAyGiqysei+L8uVPGuinDiqwlN6V0A+KUpN4Beb1evPTSS3jllVeQl5dnnLNxOBzIysqCw+HAli1b0NjYiIKCAmiahu3bt8Pj8WDDhg0AgI0bN6KiogL3338/9uzZA7/fj0cffRRer5fvcoiuYRN68F8PoF6QYbUjq6BkXttMKoBaWloAAD/4wQ8S2vft24df//rXAIBnnnkGZrMZdXV1CIfDqKmpwfPPP39x0BkZaGtrw7Zt2+DxeJCTk4P6+no8+eST8zoQIrq6wnow4QbEzPxlyEjxBsQpJhGR+Q5soYVCITgcDui6Dk3TVA+H6LonIvj87f0Y+shntC2t+AG+9b37ZvwINtfXKJ8FI6IrkvgkomPDCW3aTSvmdf4HYAAR0RxMnh/F+NkBY92UYUn62w9nwgAioisaP/clYtGwsW7PK0Jm/sy3zSSDAURElyUiOP/16YQbELMKboI5ha9gvRQDiIguTwSj/s8SmnKK3MA8z/8ADCAiuoJ4LIoJ/eJ3ek09gDrfE9AAA4iIriA8chaR0XPGeoY1E1lL5ncD4hQGEBFd1sRwAPFvnIDOKiyFJSs3LdtmABHRrEQEoVN9AC7er2zPLYTJnJ7fNGUAEdGsJDaJcOirhDZH2W1pOf8DMICI6DJikXGMD31prJsyLLDm5Kdt+wwgIpqRiGB86EvEIuNGmy07H9kFN6VtHwwgIprVhB6AxC7egJhd5E7LDYhTGEBENKuRMx8nrGcuWQaY0hcbDCAimlE8GsbE8DduQDRnIG/ZLWk7AQ0wgIhoFuHRIUzoF3+pxmzNRKZWnNZ9MICIaEZhPQCJx431rCUlab0CBjCAiGgGIoIR/6eAXAwge14hTOaMtO6HAURE00gsirFA4hPwWumtad8PA4iIpolFwwiPDhnrGbYsZBXM7yd4ZsIAIqJpzp87hcnzo8a62ZoJe15h2vfDACKiBDN9A2JucTnMlvT/bh8DiIimCX3Zl7CeVVCS9hPQAAOIiC4Rj04gMvb1xQaTOS0/wTOT9HypBxEtGmNjY4hGo7P2T46exfnhizcgwmzFWEQQGx6esT4vLw8ZGam9O2IAEd1gmpqa8PLLL8/aX7O2HFtrboX5X294PvrnGdR97y7EZ/gNZYvFgtdeew233XZbSmNhABHdYL7++mucOnVq9oI7/g1943diPOZAWeZJfPBRLwa/nLneYrFc9t3UlSR1DqilpQWrV6+GpmnQNA0ejwevv/660T8xMQGv14vCwkLk5uairq4OgUAgYRsDAwOora1FdnY2iouLsXPnTkxOTl66KyJSICPDAtuy/8DgRAWGoqXoGf13vDeQc9X2l1QAlZaWYvfu3ejq6sLx48fxwx/+ED/96U/R29sLAHjkkUfw6quvorW1FZ2dnTh9+jTuvfde4+9jsRhqa2sRiUTwzjvv4MUXX8T+/fuxa9eu9B4VEaUkLzsTeY4yABc+f8XEhvHJ9HwB/YxknpYsWSJ/+tOfZHh4WKxWq7S2thp9fX19AkB8Pp+IiBw6dEjMZrP4/X6jpqWlRTRNk3A4POd96rouAETX9fkOn+iG88tf/lJw4Vvmpy2337xM9u1+Vp7873fkiaeOyX/958tSXFw+a73FYpEPPvhg2j7m+hpN+RxQLBZDa2srxsbG4PF40NXVhWg0iurqaqNmxYoVcLvd8Pl82LBhA3w+H1atWgWn02nU1NTUYNu2bejt7cV3vvOdpMZw8uRJ5OZexXQmug7puj5rX98/g3jmf3ZjackdcN+0HKbxkzj71Rez1osIPvvsM9hsid+SODo6OstfJEo6gHp6euDxeDAxMYHc3FwcPHgQFRUV6O7uhs1mQ35+fkK90+mE33/hkp7f708In6n+qb7ZhMNhhMMXf5coFAoBuDCRPH9ElJxIJDJrXzgaw/99egb49DWYTSbABMRlhstf3zAyMoLhSy7Rj42NzWksSQfQt7/9bXR3d0PXdfztb39DfX09Ojs7k91MUpqbm/HEE09Ma6+qqoKmaVd130TXm6VLl86pLi7yzZ8Dm5HJZMKaNWtw++23J7RPvUm4kqTvhLbZbLj55ptRWVmJ5uZmrFmzBs8++yxcLhcikci0JAwEAnC5XAAAl8s17arY1PpUzUyampqg67qxDA4OJjtsIroGzftRjHg8jnA4jMrKSlitVnR0dBh9/f39GBgYgMfjAQB4PB709PQgGAwaNe3t7dA0DRUVFbPuw263G5f+pxYiWvyS+gjW1NSEe+65B263GyMjI3jppZfw9ttv480334TD4cCWLVvQ2NiIgoICaJqG7du3w+PxYMOGDQCAjRs3oqKiAvfffz/27NkDv9+PRx99FF6vF3Z7+p+0JaJrW1IBFAwG8atf/QpnzpyBw+HA6tWr8eabb+JHP/oRAOCZZ56B2WxGXV0dwuEwampq8Pzzzxt/n5GRgba2Nmzbtg0ejwc5OTmor6/Hk08+md6jIqJZZWVlpe1ThMViSfk5MAAwiVzhFPc1KBQKweFwQNd1fhwjStLQ0BDGx8evXDhHTqdz2mX4ub5G+SwY0Q2msLAQhYXp/3bDVPD7gIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyFtUDSIWIAABCoZDikRDRTKZem1Ov1dksygAaGhoCAJSVlSkeCRFdzsjICBwOx6z9izKACgoKAAADAwOXPThKFAqFUFZWhsHBQWiapno4iwLnLDUigpGREZSUlFy2blEGkNl84dSVw+Hg/xQp0DSN85Ykzlny5vLmgCehiUgZBhARKbMoA8hut+Pxxx+H3W5XPZRFhfOWPM7Z1WWSK10nIyK6ShblOyAiuj4wgIhIGQYQESnDACIiZRZlAO3duxfLly9HZmYmqqqqcOzYMdVDUqa5uRnr1q1DXl4eiouLsWnTJvT39yfUTExMwOv1orCwELm5uairq0MgEEioGRgYQG1tLbKzs1FcXIydO3dicnJyIQ9Fmd27d8NkMmHHjh1GG+dsgcgic+DAAbHZbPLnP/9Zent7ZevWrZKfny+BQED10JSoqamRffv2yYkTJ6S7u1t+/OMfi9vtltHRUaPmoYcekrKyMuno6JDjx4/Lhg0b5Lvf/a7RPzk5KStXrpTq6mr54IMP5NChQ1JUVCRNTU0qDmlBHTt2TJYvXy6rV6+Whx9+2GjnnC2MRRdA69evF6/Xa6zHYjEpKSmR5uZmhaO6dgSDQQEgnZ2dIiIyPDwsVqtVWltbjZq+vj4BID6fT0REDh06JGazWfx+v1HT0tIimqZJOBxe2ANYQCMjI3LLLbdIe3u7fP/73zcCiHO2cBbVR7BIJIKuri5UV1cbbWazGdXV1fD5fApHdu3QdR3AxQd2u7q6EI1GE+ZsxYoVcLvdxpz5fD6sWrUKTqfTqKmpqUEoFEJvb+8Cjn5heb1e1NbWJswNwDlbSIvqYdSzZ88iFosl/KMDgNPpxMmTJxWN6toRj8exY8cO3HnnnVi5ciUAwO/3w2azIT8/P6HW6XTC7/cbNTPN6VTf9ejAgQN4//338d57703r45wtnEUVQHR5Xq8XJ06cwD/+8Q/VQ7mmDQ4O4uGHH0Z7ezsyMzNVD+eGtqg+ghUVFSEjI2Pa1YhAIACXy6VoVNeGhoYGtLW14a233kJpaanR7nK5EIlEMDw8nFD/zTlzuVwzzulU3/Wmq6sLwWAQd9xxBywWCywWCzo7O/Hcc8/BYrHA6XRyzhbIogogm82GyspKdHR0GG3xeBwdHR3weDwKR6aOiKChoQEHDx7E4cOHUV5entBfWVkJq9WaMGf9/f0YGBgw5szj8aCnpwfBYNCoaW9vh6ZpqKioWJgDWUB33XUXenp60N3dbSxr167F5s2bjf/mnC0Q1WfBk3XgwAGx2+2yf/9++fDDD+XBBx+U/Pz8hKsRN5Jt27aJw+GQt99+W86cOWMs4+PjRs1DDz0kbrdbDh8+LMePHxePxyMej8fon7qkvHHjRunu7pY33nhDli5dekNdUv7mVTARztlCWXQBJCLyhz/8Qdxut9hsNlm/fr28++67qoekDIAZl3379hk158+fl9/85jeyZMkSyc7Olp/97Gdy5syZhO188cUXcs8990hWVpYUFRXJb3/7W4lGowt8NOpcGkCcs4XBr+MgImUW1TkgIrq+MICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJl/h+sfDhSwdGqZwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-829.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
